package cn.net.withub.dataCollector.web.collector;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.net.withub.dataCollector.common.model.TCollectorConfig;
import cn.net.withub.dataCollector.common.model.TCollectorData;
import cn.net.withub.dataCollector.common.utils.CreateNewKey;
import cn.net.withub.dataCollector.common.utils.HTMLSpirit;
import cn.net.withub.dataCollector.web.service.CollectorService;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.regex.Pattern;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.web.context.WebApplicationContext;

/**
 * 网易要闻
 */
public class CollectorUtilsWy extends CrawlerBase {

    public CollectorUtilsWy(String crawlPath, boolean autoParse, WebApplicationContext wac, TCollectorConfig config) {
        super(crawlPath, autoParse, wac, config);
        webAppliction = wac;
        if(webAppliction !=null) collectorService = (CollectorService) webAppliction.getBean("collectorService");
        if(config != null) tCollectorConfig = config;
        sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    }

    public void visit(Page page, CrawlDatums crawlDatums) {
        String url = page.url();
        String html = page.html();
        String contentType = page.contentType();
        String nextUrl = "";
        String title = "", fbsj = "", djcs = "", xxly = "";//信息来源
        log.info("--------------------------URL:" + url);
        try{
            if ("https://news.163.com/".equals(url)) {//国内新闻
                Elements elements = page.select("#js_top_news a");
                for (Element e : elements) {
                    nextUrl = e.attr("abs:href");
                    title = e.text();
                    TCollectorData tCollectorData = (TCollectorData) collectorService.load(TCollectorData.class, "title ='" + title + "'");
                    if (tCollectorData == null) {
                        crawlDatums.add(new CrawlDatum(nextUrl, "newsDetail").meta("title", title).meta("preUrL", url));
                    }
                }
            } else if (page.matchType("newsDetail")) {//单条新闻详情页面
                String tempStr = page.select("#epContentLeft").select(".post_time_source").text();
                fbsj = tempStr.substring(0, 19).trim();
                xxly = tempStr.substring(tempStr.indexOf("来源:") + 3).trim();
                Element contentEle = page.select("#endText",0);
                contentEle.select(".ep-source").remove();
                //类似于专题  回顾  推荐,去掉
                if (contentEle.select("pre").size() > 0) {
                    contentEle.select("pre").remove();
                }
                String content = contentEle.html();
                String text = HTMLSpirit.getTextFromTHML(content);
                if("".equals(text.trim())) return;//无文字  基本全是视频
                if(content.contains("type=\"video/mp4\"")){
                    //log.info("包含视频资源");
                    return;
                }
                title = getStr(page.meta("title"));
                String preUrL = getStr(page.meta("preUrL"));
                Elements imgs = contentEle.select("img[src]");
                String id = new CreateNewKey().createId();
                //遍历所有html中所有图片
                for (Element img : imgs) {
                    String src = img.attr("abs:src");
                    //此处图片地址包含中文会报错
                    src = fmtUrl(src);
                    //网易图标
                    if (Pattern.matches("http.*cnews/css13/img/end_news.png",src)) {
                        log.info("图标src:" + src);
                        continue;
                    }
                    //img.attr("style", "display: block;margin : 0 auto");
                    //img.parent().attr("style", "text-align: center;");
                    String imageName = "";
                    String serverFilePath = "";
                    try {
                        if(src.contains("?")){
                            serverFilePath = FtpUtil.getImage(src, "wy");
                            imageName = src.substring(src.lastIndexOf("/") + 1, src.lastIndexOf("?"));
                        }else {
                            serverFilePath = FtpUtil.getImage(src, "1");
                            imageName = src.substring(src.lastIndexOf("/") + 1, src.length());
                        }
                        //img.attr("src", serverFilePath);
                        saveElement(id, imageName, src, serverFilePath);
                    } catch (IOException e) {
                        log.error("出错src:" + src + ";父级路径:" + url);
                        e.printStackTrace();
                    }
                }
                int zt = 1;
                try {
                    //保存TCollectorData
                    saveData(id, title, fbsj, xxly, "html", url, "", "", "", "");
                    //保存TCollectorContent
                    saveContent(id, content, text);
                    //保存最大发布时间
                    saveMaxFbsj(fbsj);
                    zt = 1;
                } catch (Exception e) {
                    zt = 0;
                    e.printStackTrace();
                } finally {
                    saveLog(title, fbsj, preUrL, url, zt);
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }

}
